{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"-1\"\n",
    "\n",
    "from warmup_cosine_decay_scheduler import WarmUpCosineDecayScheduler\n",
    "learning_rate = 1e-3\n",
    "sample_count =5760\n",
    "epochs = 50\n",
    "warmup_epoch = 10\n",
    "batch_size = 32\n",
    "learning_rate_base = learning_rate\n",
    "total_steps = int(epochs * sample_count / batch_size)\n",
    "warmup_steps = int(warmup_epoch * sample_count / batch_size)\n",
    "\n",
    "warm_up_lr = WarmUpCosineDecayScheduler(learning_rate_base=learning_rate_base,\n",
    "                                        total_steps=total_steps,\n",
    "                                        warmup_learning_rate=0,\n",
    "                                        warmup_steps=warmup_steps,\n",
    "                                        hold_base_rate_steps=0,\n",
    "                                        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "# Create a model.\n",
    "model = Sequential()\n",
    "# model.add(Dense(32, activation='relu', input_dim=100))\n",
    "model.add(Dense(10, activation='softmax'))\n",
    "model.compile(optimizer='rmsprop',\n",
    "            loss='categorical_crossentropy',\n",
    "            metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tensorflow import keras\n",
    "from keras import backend as K\n",
    "data = np.random.random((sample_count, 100))\n",
    "labels = np.random.randint(10, size=(sample_count, 1))\n",
    "\n",
    "one_hot_labels = keras.utils.to_categorical(labels, num_classes=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "Epoch 1/100\n",
      "5760/5760 [==============================] - 0s 72us/step - loss: 2.5339 - acc: 0.1016\n",
      "Epoch 2/100\n",
      "5760/5760 [==============================] - 0s 68us/step - loss: 2.3903 - acc: 0.0979\n",
      "Epoch 3/100\n",
      "5760/5760 [==============================] - 0s 70us/step - loss: 2.3561 - acc: 0.0991\n",
      "Epoch 4/100\n",
      "5760/5760 [==============================] - 0s 69us/step - loss: 2.3532 - acc: 0.1028\n",
      "Epoch 5/100\n",
      "5760/5760 [==============================] - 0s 71us/step - loss: 2.3492 - acc: 0.1026\n",
      "Epoch 6/100\n",
      "5760/5760 [==============================] - 0s 71us/step - loss: 2.3457 - acc: 0.1017\n",
      "Epoch 7/100\n",
      "5760/5760 [==============================] - 0s 69us/step - loss: 2.3390 - acc: 0.1090\n",
      "Epoch 8/100\n",
      "5760/5760 [==============================] - 0s 71us/step - loss: 2.3339 - acc: 0.1071\n",
      "Epoch 9/100\n",
      "5760/5760 [==============================] - 0s 69us/step - loss: 2.3271 - acc: 0.1106\n",
      "Epoch 10/100\n",
      "5760/5760 [==============================] - 0s 64us/step - loss: 2.3209 - acc: 0.1135\n",
      "Epoch 11/100\n",
      "5760/5760 [==============================] - 0s 74us/step - loss: 2.3120 - acc: 0.1175\n",
      "Epoch 12/100\n",
      "5760/5760 [==============================] - 0s 74us/step - loss: 2.3076 - acc: 0.1253\n",
      "Epoch 13/100\n",
      "5760/5760 [==============================] - 0s 72us/step - loss: 2.3005 - acc: 0.1208\n",
      "Epoch 14/100\n",
      "5760/5760 [==============================] - 0s 72us/step - loss: 2.2947 - acc: 0.1288\n",
      "Epoch 15/100\n",
      "5760/5760 [==============================] - 0s 70us/step - loss: 2.2889 - acc: 0.1314\n",
      "Epoch 16/100\n",
      "5760/5760 [==============================] - 0s 76us/step - loss: 2.2831 - acc: 0.1351\n",
      "Epoch 17/100\n",
      "5760/5760 [==============================] - 0s 71us/step - loss: 2.2797 - acc: 0.1429\n",
      "Epoch 18/100\n",
      "5760/5760 [==============================] - 0s 70us/step - loss: 2.2761 - acc: 0.1420\n",
      "Epoch 19/100\n",
      "5760/5760 [==============================] - 0s 64us/step - loss: 2.2716 - acc: 0.1410\n",
      "Epoch 20/100\n",
      "5760/5760 [==============================] - 0s 69us/step - loss: 2.2688 - acc: 0.1517\n",
      "Epoch 21/100\n",
      "5760/5760 [==============================] - 0s 69us/step - loss: 2.2656 - acc: 0.1497\n",
      "Epoch 22/100\n",
      "5760/5760 [==============================] - 0s 64us/step - loss: 2.2610 - acc: 0.1516\n",
      "Epoch 23/100\n",
      "5760/5760 [==============================] - 0s 68us/step - loss: 2.2590 - acc: 0.1543\n",
      "Epoch 24/100\n",
      "5760/5760 [==============================] - 0s 61us/step - loss: 2.2574 - acc: 0.1589\n",
      "Epoch 25/100\n",
      "5760/5760 [==============================] - 0s 63us/step - loss: 2.2566 - acc: 0.1620\n",
      "Epoch 26/100\n",
      "5760/5760 [==============================] - 0s 66us/step - loss: 2.2537 - acc: 0.1575\n",
      "Epoch 27/100\n",
      "5760/5760 [==============================] - 0s 67us/step - loss: 2.2523 - acc: 0.1627\n",
      "Epoch 28/100\n",
      "5760/5760 [==============================] - 0s 67us/step - loss: 2.2500 - acc: 0.1665\n",
      "Epoch 29/100\n",
      "5760/5760 [==============================] - 0s 75us/step - loss: 2.2489 - acc: 0.1589\n",
      "Epoch 30/100\n",
      "5760/5760 [==============================] - 0s 70us/step - loss: 2.2470 - acc: 0.1672\n",
      "Epoch 31/100\n",
      "5760/5760 [==============================] - 0s 74us/step - loss: 2.2465 - acc: 0.1656\n",
      "Epoch 32/100\n",
      "5760/5760 [==============================] - 0s 69us/step - loss: 2.2449 - acc: 0.1681\n",
      "Epoch 33/100\n",
      "5760/5760 [==============================] - 0s 69us/step - loss: 2.2442 - acc: 0.1630\n",
      "Epoch 34/100\n",
      "5760/5760 [==============================] - 0s 73us/step - loss: 2.2425 - acc: 0.1701\n",
      "Epoch 35/100\n",
      "5760/5760 [==============================] - 0s 68us/step - loss: 2.2427 - acc: 0.1663\n",
      "Epoch 36/100\n",
      "5760/5760 [==============================] - 0s 71us/step - loss: 2.2413 - acc: 0.1703\n",
      "Epoch 37/100\n",
      "5760/5760 [==============================] - 0s 70us/step - loss: 2.2406 - acc: 0.1745\n",
      "Epoch 38/100\n",
      "5760/5760 [==============================] - 0s 67us/step - loss: 2.2394 - acc: 0.1703\n",
      "Epoch 39/100\n",
      "5760/5760 [==============================] - 0s 69us/step - loss: 2.2396 - acc: 0.1710\n",
      "Epoch 40/100\n",
      "5760/5760 [==============================] - 0s 71us/step - loss: 2.2388 - acc: 0.1733\n",
      "Epoch 41/100\n",
      "5760/5760 [==============================] - 0s 70us/step - loss: 2.2383 - acc: 0.1712\n",
      "Epoch 42/100\n",
      "5760/5760 [==============================] - 0s 72us/step - loss: 2.2379 - acc: 0.1766\n",
      "Epoch 43/100\n",
      "5760/5760 [==============================] - 0s 70us/step - loss: 2.2375 - acc: 0.1724\n",
      "Epoch 44/100\n",
      "5760/5760 [==============================] - 0s 76us/step - loss: 2.2371 - acc: 0.1741\n",
      "Epoch 45/100\n",
      "5760/5760 [==============================] - 0s 66us/step - loss: 2.2368 - acc: 0.1740\n",
      "Epoch 46/100\n",
      "5760/5760 [==============================] - 0s 69us/step - loss: 2.2366 - acc: 0.1747\n",
      "Epoch 47/100\n",
      "5760/5760 [==============================] - 0s 68us/step - loss: 2.2364 - acc: 0.1745\n",
      "Epoch 48/100\n",
      "5760/5760 [==============================] - 0s 73us/step - loss: 2.2362 - acc: 0.1755\n",
      "Epoch 49/100\n",
      "5760/5760 [==============================] - 0s 74us/step - loss: 2.2361 - acc: 0.1748\n",
      "Epoch 50/100\n",
      "5760/5760 [==============================] - 0s 70us/step - loss: 2.2372 - acc: 0.1740\n",
      "Epoch 51/100\n",
      "5760/5760 [==============================] - 0s 72us/step - loss: 2.2372 - acc: 0.1743\n",
      "Epoch 52/100\n",
      "5760/5760 [==============================] - 0s 74us/step - loss: 2.2369 - acc: 0.1736\n",
      "Epoch 53/100\n",
      "5760/5760 [==============================] - 0s 70us/step - loss: 2.2368 - acc: 0.1743\n",
      "Epoch 54/100\n",
      "5760/5760 [==============================] - 0s 74us/step - loss: 2.2368 - acc: 0.1733\n",
      "Epoch 55/100\n",
      "5760/5760 [==============================] - 0s 69us/step - loss: 2.2366 - acc: 0.1745\n",
      "Epoch 56/100\n",
      "5760/5760 [==============================] - 0s 67us/step - loss: 2.2365 - acc: 0.1769\n",
      "Epoch 57/100\n",
      "5760/5760 [==============================] - 0s 69us/step - loss: 2.2363 - acc: 0.1766\n",
      "Epoch 58/100\n",
      "5760/5760 [==============================] - 0s 65us/step - loss: 2.2360 - acc: 0.1780\n",
      "Epoch 59/100\n",
      "5760/5760 [==============================] - 0s 71us/step - loss: 2.2360 - acc: 0.1752\n",
      "Epoch 60/100\n",
      "5760/5760 [==============================] - 0s 69us/step - loss: 2.2358 - acc: 0.1764\n",
      "Epoch 61/100\n",
      "5760/5760 [==============================] - 0s 67us/step - loss: 2.2357 - acc: 0.1750\n",
      "Epoch 62/100\n",
      "5760/5760 [==============================] - 0s 75us/step - loss: 2.2356 - acc: 0.1762\n",
      "Epoch 63/100\n",
      "5760/5760 [==============================] - 0s 71us/step - loss: 2.2354 - acc: 0.1771\n",
      "Epoch 64/100\n",
      "5760/5760 [==============================] - 0s 68us/step - loss: 2.2353 - acc: 0.1762\n",
      "Epoch 65/100\n",
      "5760/5760 [==============================] - 0s 66us/step - loss: 2.2352 - acc: 0.1773\n",
      "Epoch 66/100\n",
      "5760/5760 [==============================] - 0s 69us/step - loss: 2.2350 - acc: 0.1780\n",
      "Epoch 67/100\n",
      "5760/5760 [==============================] - 0s 71us/step - loss: 2.2349 - acc: 0.1771\n",
      "Epoch 68/100\n",
      "5760/5760 [==============================] - 0s 68us/step - loss: 2.2349 - acc: 0.1773\n",
      "Epoch 69/100\n",
      "5760/5760 [==============================] - 0s 64us/step - loss: 2.2348 - acc: 0.1781\n",
      "Epoch 70/100\n",
      "5760/5760 [==============================] - 0s 69us/step - loss: 2.2347 - acc: 0.1776\n",
      "Epoch 71/100\n",
      "5760/5760 [==============================] - 0s 71us/step - loss: 2.2346 - acc: 0.1781\n",
      "Epoch 72/100\n",
      "5760/5760 [==============================] - 0s 71us/step - loss: 2.2346 - acc: 0.1774\n",
      "Epoch 73/100\n",
      "72     ???? \n",
      "5760/5760 [==============================] - 0s 74us/step - loss: 2.2347 - acc: 0.1776\n",
      "Epoch 74/100\n",
      "5760/5760 [==============================] - 0s 69us/step - loss: 2.2347 - acc: 0.1771\n",
      "Epoch 75/100\n",
      "5760/5760 [==============================] - 0s 68us/step - loss: 2.2347 - acc: 0.1780\n",
      "Epoch 76/100\n",
      "5760/5760 [==============================] - 0s 59us/step - loss: 2.2347 - acc: 0.1776\n",
      "Epoch 77/100\n",
      "5760/5760 [==============================] - 0s 71us/step - loss: 2.2346 - acc: 0.1778\n",
      "Epoch 78/100\n",
      "5760/5760 [==============================] - 0s 70us/step - loss: 2.2346 - acc: 0.1780\n",
      "Epoch 79/100\n",
      "5760/5760 [==============================] - 0s 70us/step - loss: 2.2346 - acc: 0.1780\n",
      "Epoch 80/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5760/5760 [==============================] - 0s 58us/step - loss: 2.2346 - acc: 0.1778\n",
      "Epoch 81/100\n",
      "5760/5760 [==============================] - 0s 68us/step - loss: 2.2345 - acc: 0.1778\n",
      "Epoch 82/100\n",
      "5760/5760 [==============================] - 0s 67us/step - loss: 2.2345 - acc: 0.1780\n",
      "Epoch 83/100\n",
      "5760/5760 [==============================] - 0s 73us/step - loss: 2.2345 - acc: 0.1778\n",
      "Epoch 84/100\n",
      "72     ???? \n",
      "5760/5760 [==============================] - 0s 67us/step - loss: 2.2345 - acc: 0.1778\n",
      "Epoch 85/100\n",
      "5760/5760 [==============================] - 0s 69us/step - loss: 2.2345 - acc: 0.1778\n",
      "Epoch 86/100\n",
      "5760/5760 [==============================] - 0s 69us/step - loss: 2.2345 - acc: 0.1778\n",
      "Epoch 87/100\n",
      "5760/5760 [==============================] - 0s 61us/step - loss: 2.2345 - acc: 0.1778\n",
      "Epoch 88/100\n",
      "5760/5760 [==============================] - 0s 71us/step - loss: 2.2345 - acc: 0.1780\n",
      "Epoch 89/100\n",
      "5760/5760 [==============================] - 0s 65us/step - loss: 2.2345 - acc: 0.1778\n",
      "Epoch 90/100\n",
      "5760/5760 [==============================] - 0s 74us/step - loss: 2.2345 - acc: 0.1778\n",
      "Epoch 91/100\n",
      "5760/5760 [==============================] - 0s 72us/step - loss: 2.2345 - acc: 0.1778\n",
      "Epoch 92/100\n",
      "5760/5760 [==============================] - 0s 68us/step - loss: 2.2345 - acc: 0.1778\n",
      "Epoch 93/100\n",
      "5760/5760 [==============================] - 0s 70us/step - loss: 2.2345 - acc: 0.1778\n",
      "Epoch 94/100\n",
      "5760/5760 [==============================] - 0s 69us/step - loss: 2.2345 - acc: 0.1778\n",
      "Epoch 95/100\n",
      "5760/5760 [==============================] - 0s 64us/step - loss: 2.2345 - acc: 0.1778\n",
      "Epoch 96/100\n",
      "5760/5760 [==============================] - 0s 67us/step - loss: 2.2345 - acc: 0.1778\n",
      "Epoch 97/100\n",
      "5760/5760 [==============================] - 0s 64us/step - loss: 2.2345 - acc: 0.1778\n",
      "Epoch 98/100\n",
      "5760/5760 [==============================] - 0s 69us/step - loss: 2.2345 - acc: 0.1778\n",
      "Epoch 99/100\n",
      "5760/5760 [==============================] - 0s 69us/step - loss: 2.2345 - acc: 0.1778\n",
      "Epoch 100/100\n",
      "5760/5760 [==============================] - 0s 72us/step - loss: 2.2345 - acc: 0.1778\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f5d61a37b70>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.callbacks import TensorBoard, Callback, ModelCheckpoint\n",
    "\n",
    "log_local = './logs/'+'L0002_test_lr'\n",
    "tensorBoard = TensorBoard(log_dir=log_local)\n",
    "model.fit(data, one_hot_labels, epochs=100, batch_size=batch_size,\n",
    "        verbose=1, callbacks=[warm_up_lr,tensorBoard])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1.0,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1.0,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
